/****************************************************************************/
/* Batch System Management with performance prediction: OAR v2.x            */
/*                                                                          */
/* Author(s):                                                               */
/*    - Yves Caniou (yves.caniou@ens-lyon.fr)                               */
/*    - Jonathan Amiez (jonathan.amiez@gmail.com)                           */
/*                                                                          */
/* $LICENSE$                                                                */
/****************************************************************************/

#include "debug.hh"
#include "OAR2_xBatchSystem.hh"
#include "Parsers.hh"
#include <fcntl.h>       // for O_RDONLY
#include <unistd.h>      // for read()

const char * const OAR2_xBatchSystem::statusNames[] = {
  "Error",
  "Error",
  "Terminated",
  "Running",
  "Waiting",
  "Launching",
} ;

OAR2_xBatchSystem::OAR2_xBatchSystem(int ID, const char * batchname)
{
  if( pathToNFS == NULL ) {
    ERROR_EXIT("OAR needs a path to a NFS directory to store its script\n") ;
  }
#if defined YC_DEBUG
  TRACE_TEXT(TRACE_ALL_STEPS,"Nom NFS: " << getNFSPath() << "\n") ;
#endif

  if( pathToTmp == NULL ) {
    ERROR_EXIT("OAR needs a path to a tmp directory to store its script\n") ;
  }
#if defined YC_DEBUG
  TRACE_TEXT(TRACE_ALL_STEPS,"Nom Tmp: " << getTmpPath() << "\n") ;
#endif

  batch_ID = ID ;
  batchName = batchname ;
  /* Dirty Trick for OAR1.6 to get information on default queue */
  internQueueName = strdup((char*)
    Parsers::Results::getParamValue(Parsers::Results::INTERNOARQUEUENAME)) ;
#if defined YC_DEBUG
  if( internQueueName != NULL )
    TRACE_TEXT(TRACE_ALL_STEPS,"Nom queue interne: " << internQueueName 
  	       << "\n") ;
#endif
  
  shell = BatchSystem::emptyString ;
  prefixe = "#!/bin/sh" ;
  postfixe = BatchSystem::emptyString ;

  nodesNumber = "#OAR -l nodes=" ;
  serial = "#OAR -l nodes=1" ;
  coresNumber = "/cores=" ;
  walltime = ",walltime=" ;
  submittingQueue = "#OAR -q " ;
  minimumMemoryUsed = BatchSystem::emptyString ;
  
  mail = BatchSystem::emptyString ;
  account = BatchSystem::emptyString ;
  setSTDOUT = BatchSystem::emptyString ;
  setSTDIN = BatchSystem::emptyString ;
  setSTDERR = BatchSystem::emptyString ;
 
  /* cd, to be sure that OAR takes PWD and not /bin/pwd: for Grenoble */
  submitCommand = "cd . ; oarsub -S" ;
  killCommand = "oardel " ;
  wait4Command = "oarstat -X -j" ;
  waitFilter = "grep state | cut -d \"<\" -f 2 | cut -d \">\" -f 2" ;
  exitCode = "0" ;
  loadCommand = "oarstat -Xf" ;
  
  jid_extract_patterns = "grep \"OAR_JOB_ID=\" | cut -d \"=\" -f 2" ;

  /* OAR behaves with SQL scripts to reserve specials nodes */
  /* but the following line is not good enough: too less nodes */
  /*    ELBASE_NODETYPE,"#OAR -p \"hostname='%s'\"", */

  /* Information for META_VARIABLES */
  batchJobID = "$OAR_JOBID" ;
  nodeFileName = "$OAR_NODEFILE" ;
  nodeIdentities = "cat $OAR_NODEFILE" ;  
}

OAR2_xBatchSystem::~OAR2_xBatchSystem()
{
}

/*********************** Job Managing ******************************/

BatchSystem::batchJobState
OAR2_xBatchSystem::askBatchJobStatus(int batchJobID)
{
  char * filename ;
  int file_descriptor ;
  char * chaine ;
  int i=0 ;
  int nbread ;
  batchJobState status ;
  
  /* If job has completed, not ask batch system */
  status = getRecordedBatchJobStatus( batchJobID ) ;
  if( (status == TERMINATED) || (status == CANCELED) || (status == ERROR) )
    return status ;
  /* create a temporary file to get results and batch job ID */
  filename = createUniqueTemporaryTmpFile("DIET_batch_finish") ;
  file_descriptor = open(filename,O_RDONLY) ;
  if( file_descriptor == -1 ) {
    ERROR("Cannot open file", UNDETERMINED ) ;
  }

  /* Ask batch system the job status */      
  chaine = (char*)malloc(sizeof(char)*(strlen(wait4Command)
				       + NBDIGITS_MAX_BATCH_ID
				       + strlen(waitFilter)
				       + strlen(filename)
				       + 7 + 1) ) ;
  sprintf(chaine,"%s %d | %s > %s",
	  wait4Command,batchJobID,waitFilter,filename) ;
#if defined YC_DEBUG
  TRACE_TEXT(TRACE_ALL_STEPS,"Execute:\n " << chaine << "\n") ;
#endif
  if( system(chaine) != 0 ) {
    ERROR("Cannot submit script", NB_STATUS) ;
  }
  /* Get job status */  
  for( int i = 0 ; i<=NBDIGITS_MAX_BATCH_ID ; i++ )
    chaine[i] = '\0' ;
  if( (nbread=readn(file_descriptor,chaine,NBDIGITS_MAX_JOB_STATUS))
      == 0 ) {
    ERROR("Error with I/O file. Cannot read the batch status", NB_STATUS) ;
  }
  /* Adjust what have been read */
  if( chaine[nbread-1] == '\n' )
    chaine[nbread-1] = '\0' ;
  while( (i<NB_STATUS) && 
	 (strcmp(chaine,OAR2_xBatchSystem::statusNames[i])!=0) ) {
    i++ ;
  }
  
  if( i==NB_STATUS ) {
    ERROR("Cannot get batch job " << batchJobID << " status: " << chaine
	  << "\n",NB_STATUS) ;
  }
  /* Remove temporary file by closing it */
#if REMOVE_BATCH_TEMPORARY_FILE
  unlink( filename ) ;
#endif
  if( close(file_descriptor) != 0 ) {
    WARNING("Couln't remove I/O redirection file") ;
  }
  updateBatchJobStatus(batchJobID,(batchJobState)i) ;
  free(chaine) ;
  free(filename) ;
  return (batchJobState)i ;
}

int
OAR2_xBatchSystem::isBatchJobCompleted(int batchJobID)
{
  batchJobState status = getRecordedBatchJobStatus(batchJobID) ;
  
  if( (status == TERMINATED) || (status == CANCELED) || (status == ERROR) )
    return 1 ;
  status = askBatchJobStatus(batchJobID) ;
  if( (status == TERMINATED) || (status == CANCELED) || (status == ERROR) )
    return 1 ;
  else if( status == NB_STATUS )
    return -1 ;
  return 0 ;
}

/************************ Submitting Funtions *******************/

int
OAR2_xBatchSystem::diet_submit_parallel(diet_profile_t * profile,
				  const char * command,
				  const char * user_batch_cmd)
{
  int nbread ;
  char * script = NULL ;
  char small_chaine[50] ; // This must be gt NBDIGITS_MAX_BATCH_ID
  char * chaine ; 
  char * options ;
  int file_descriptor ;
  int file_descriptor_2 ;
  char * filename ;
  char * filename_2 ;

  unsigned long heures;
  int minutes;
  char* wall_time = NULL;

  /* Options, if available, are in the following order:
     nbnodes
     walltime
     queue
     minimumMemoryUsed
     mail
     stdin
     stdout
     stderr
  */
  
  if( profile->walltime > 59 ) {
    heures = profile->walltime / 60 ;
    minutes = profile->walltime % 60 ;
  }
  else {
     heures = 0 ;
     minutes = profile->walltime ;
  }

  wall_time = ( char* )malloc( ( sizeof( unsigned long ) * 8 + 1 ) 
		  + sizeof( int ) + ( sizeof( char ) * 4 ) ) ;
  if( wall_time == NULL ) {
     perror("error allocating memory when building script (walltime)... \n"
            "Service not launched\n\n");
  }
  sprintf( wall_time,
           "%ld:%d:00",
           heures, minutes // seconds are not used
         );


  options = (char*)calloc(9000,sizeof(char)) ; /* FIXME: Reduce size */
  if( options == NULL )  {
    ERROR("error allocating memory when building script (options)... \n"
	  "Service not launched\n\n", -1);
  }
  
  /* TODO: update diet_profile in order to provide the number of core 
   * the application can use.
   * SEE : this->coresNumber, BatchSystem->coresNumber. */
  if( profile->parallel_flag == 1 )
    sprintf(options,
	    "%s%s%s\n",
	    this->serial, this->walltime, wall_time) ;
  else
    sprintf(options,
	    "%s%d%s%s\n",
	    this->nodesNumber, profile->nbprocs,
	    this->walltime, wall_time) ;
  sprintf(options+strlen(options),
	  "%s%s\n",
	  submittingQueue, batchQueueName) ;

  /* TODO: the user will be able to set the shell, mail, stdin, etc. */
  // Shell semble ne pas marcher... :?
  //  if( shell != emptyString ) /* & if user hasnt specified one or set to user */
  //  sprintf(options+strlen(options),
  //	    "%s bash\n",
  //	    shell ) ;

  /*
  if( setSTDIN != emptyString ) ** & if user has set one entry
  sprintf(options+strlen(options),
  "%s %s
  */

  // Build Script, copy it on the NFS path and launch in a system()
  // record the pid of the batch job

  if( user_batch_cmd == NULL )
    user_batch_cmd = emptyString;

  script = (char*)malloc(sizeof(char)*(100
				       + strlen(prefixe)
				       + strlen(options)
				       + strlen(user_batch_cmd)
				       + strlen(postfixe)
				       + 300
				       + strlen(command))) ;
  if( script == NULL ) {
    ERROR("error allocating memory when building script... \n"
	  "Service not launched\n\n", -1);
  }

  sprintf(script,
          "%s\n"
	  "%s\n"
	  ,prefixe
	  ,options
	  ) ;

  if( user_batch_cmd != emptyString )
    sprintf(script + strlen(script),
            "%s\n"
	    ,user_batch_cmd
	    ) ;

  sprintf(script + strlen(script),   
	  "%s\n"
	  "DIET_BATCH_NODESFILE=%s\n"
	  "DIET_BATCH_NODESLIST=`cat %s | sort | uniq`\n"
	  "DIET_BATCH_NBNODES=`echo $DIET_BATCH_NODESLIST | wc -l`\n"
	  "\n%s\n"
	  ,postfixe
	  ,nodeFileName
	  ,nodeFileName
	  ,command
	  ) ;
  
  /* Replace DIET meta-variable in SeD programmer's command */
  sprintf(small_chaine,"%d",profile->nbprocs) ;
  replaceAllOccurencesInString(&script,"$DIET_BATCH_NBNODES",small_chaine) ;
  sprintf(small_chaine,"%d",profile->nbprocess) ;
  replaceAllOccurencesInString(&script,"$DIET_USER_NBPROCS",small_chaine) ;
  replaceAllOccurencesInString(&script,"$DIET_NAME_FRONTALE",frontalName) ;
  replaceAllOccurencesInString(&script,"$DIET_BATCH_JOB_ID",batchJobID) ;
  replaceAllOccurencesInString(&script,"$DIET_BATCHNAME",batchName) ;

  /* Create temporary file to save the script, and submit it */
  filename = (char*)malloc(sizeof(char)*strlen(pathToNFS) + 30 ) ;
  sprintf(filename, "%sDIET_batch_script.XXXXXX", pathToNFS) ;

  file_descriptor = mkstemp( filename ) ;
  if( file_descriptor == -1 ) {
    ERROR("Cannot create batch file", -1) ;
  }

#if defined YC_DEBUG
  TRACE_TEXT(TRACE_MAIN_STEPS,"Nom script: " << filename << "\n") ;
#endif
  
  if( writen(file_descriptor, script, strlen(script)) != strlen(script) ) {
    ERROR("Cannot write the batch script on the filesystem",-1) ;
  }
  /* Make the script runnable (OAR) */
  if( chmod(filename,S_IRUSR | S_IWUSR | S_IXUSR) != 0 ) {
    WARNING("Execution rights have not been set on the batch script."
	    " This can lead to an error") ;
  }

  /***************** Submit script and store the batchID ********************/
  /* create a temporary file to get results and batch job ID */
  filename_2 = (char*)malloc(sizeof(char)*strlen(pathToTmp) + 30 ) ;
  sprintf(filename_2,"%sDIET_batch_launch.XXXXXX", pathToTmp) ;
  file_descriptor_2 = mkstemp( filename_2 ) ;
  if( file_descriptor_2 == -1 ) {
    ERROR("Cannot create batch I/O redirection file."
	  " Verify that tmp path is ok\n",-1) ;
  }

#if defined YC_DEBUG
  TRACE_TEXT(TRACE_MAIN_STEPS,
	     "Fichier pour l'ID du job batch : " << filename_2 << "\n") ; 
#endif
  /* Submit and grep the jobID */
  chaine = (char*)malloc(sizeof(char)*(strlen(submitCommand)
				       + strlen(filename)
				       + strlen(jid_extract_patterns)
				       + strlen(filename_2)
				       + 7 + 1 ) ) ;
  sprintf(chaine,"%s %s | %s > %s",
	  submitCommand,filename,jid_extract_patterns,filename_2) ;
#if defined YC_DEBUG
  TRACE_TEXT(TRACE_MAIN_STEPS,
	     "Submit avec la ligne :\n" << chaine << "\n\n") ;
#endif

  if( system(chaine) == -1 ) {
    ERROR("Cannot submit script", -1) ;
  }

  file_descriptor_2 = open(filename_2,O_RDONLY) ;
  if( file_descriptor_2 == -1 ) {
    ERROR("Cannot open batch I/O redirection file",-1) ;
  }
  /* Get batch Job ID */  
  for( int i = 0 ; i<=NBDIGITS_MAX_BATCH_ID ; i++ )
    small_chaine[i] = '\0' ;
  if( (nbread=readn(file_descriptor_2,small_chaine,NBDIGITS_MAX_BATCH_ID))
      == 0 ) {
    ERROR("Error during submission or with I/O file."
	  " Cannot read the batch ID", -1) ;
  }
  /* Just in case */
  if( small_chaine[nbread-1] == '\n' )
    small_chaine[nbread-1] = '\0' ;
  /* Store the JobID in correlation with DIET_taskID */
  if( storeBatchJobID(atoi(small_chaine), profile->dietReqID, filename) < 0 ) {
    ERROR("Not enough memory to store new batch information\n",-1) ;
  }

  /* Remove temporary files by closing them */
#if REMOVE_BATCH_TEMPORARY_FILE
  unlink( filename_2 ) ;
#endif
  if( close(file_descriptor) != 0 ) {
    WARNING("Couln't close batch script file") ;
  }
  if( close(file_descriptor_2) != 0 ) {
    WARNING("Couln't close I/O redirection file") ;
  }

  /* Free memory */
  free(wall_time);
  free(options) ;
  free(script) ;
  free(filename_2) ;
  
  return 0 ;
}


/********** Batch static information accessing Functions **********/

int
OAR2_xBatchSystem::getNbTotResources()
{
  return launchCommandAndGetInt( "oarnodes -s | grep \":\" | wc -l",
				 "DIET_getNbResources") ;
}

/* TODO: this function should be C++ written 
   or, as OAR relies on Perl, use a Perl script which has to be
   deployed
*/
int
OAR2_xBatchSystem::getNbResources() /* in the queue interQueueName */
{
  char chaine[125] ;
    
  if( internQueueName == NULL ) {
    WARNING("No internal queue Name given: use total information\n\n") ;    
    return getNbTotResources() ;
  }

  sprintf(chaine, "oarnodes | grep %s | wc -l", internQueueName) ;
  return launchCommandAndGetInt(chaine,
				"DIET_getNameResources") ;
}

char *
OAR2_xBatchSystem::getResourcesName()
{
  char chaine[150] ;
  
  sprintf(chaine, "oarnodes | grep %s | cut -d \"=\" -f 14"
	      " | cut -d \",\" -f 1", internQueueName) ;
  return launchCommandAndGetResultFilename(chaine,
					   "DIET_getNameResources") ;
}

int
OAR2_xBatchSystem::getMaxWalltime()
{
  INTERNAL_WARNING(__FUNCTION__ << " not yet implemented\n\n") ;
  return 500 ;
}

int
OAR2_xBatchSystem::getMaxProcs()
{
  INTERNAL_WARNING(__FUNCTION__ << " not yet implemented\n\n") ;
  return getNbResources() ;
}

/********** Batch dynamic information accessing Functions *********/

int
OAR2_xBatchSystem::getNbTotFreeResources()
{
  INTERNAL_WARNING(__FUNCTION__ << " not  implemented\n\n") ;
  return getNbResources() ;
}

int
OAR2_xBatchSystem::getNbFreeResources()
{
  INTERNAL_WARNING(__FUNCTION__ << " not  implemented\n\n") ;
  return getNbResources() ;
}

/*************************** Performance Prediction *************************/

char*
OAR2_xBatchSystem::getLoad() {

    char* loadfile;

    loadfile = launchCommandAndGetResultFilename(loadCommand, "OAR_load");

#if defined YC_DEBUG
  TRACE_TEXT(TRACE_MAIN_STEPS,
             "Loadfile :\n" << loadfile << "\n\n") ;
#endif
    return loadfile;
}

